{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:98: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/60 | Batch 0/77 | train_loss: 3.398 | test_loss: 3.394\n",
      "Epoch 1/60 | Batch 50/77 | train_loss: 2.747 | test_loss: 2.734\n",
      "Epoch 2/60 | Batch 0/77 | train_loss: 2.348 | test_loss: 2.336\n",
      "Epoch 2/60 | Batch 50/77 | train_loss: 1.878 | test_loss: 1.858\n",
      "Epoch 3/60 | Batch 0/77 | train_loss: 1.728 | test_loss: 1.692\n",
      "Epoch 3/60 | Batch 50/77 | train_loss: 1.471 | test_loss: 1.424\n",
      "Epoch 4/60 | Batch 0/77 | train_loss: 1.351 | test_loss: 1.309\n",
      "Epoch 4/60 | Batch 50/77 | train_loss: 1.202 | test_loss: 1.124\n",
      "Epoch 5/60 | Batch 0/77 | train_loss: 1.084 | test_loss: 1.044\n",
      "Epoch 5/60 | Batch 50/77 | train_loss: 0.988 | test_loss: 0.915\n",
      "Epoch 6/60 | Batch 0/77 | train_loss: 0.893 | test_loss: 0.860\n",
      "Epoch 6/60 | Batch 50/77 | train_loss: 0.821 | test_loss: 0.770\n",
      "Epoch 7/60 | Batch 0/77 | train_loss: 0.745 | test_loss: 0.726\n",
      "Epoch 7/60 | Batch 50/77 | train_loss: 0.696 | test_loss: 0.655\n",
      "Epoch 8/60 | Batch 0/77 | train_loss: 0.632 | test_loss: 0.615\n",
      "Epoch 8/60 | Batch 50/77 | train_loss: 0.599 | test_loss: 0.560\n",
      "Epoch 9/60 | Batch 0/77 | train_loss: 0.541 | test_loss: 0.531\n",
      "Epoch 9/60 | Batch 50/77 | train_loss: 0.509 | test_loss: 0.479\n",
      "Epoch 10/60 | Batch 0/77 | train_loss: 0.462 | test_loss: 0.458\n",
      "Epoch 10/60 | Batch 50/77 | train_loss: 0.434 | test_loss: 0.419\n",
      "Epoch 11/60 | Batch 0/77 | train_loss: 0.394 | test_loss: 0.401\n",
      "Epoch 11/60 | Batch 50/77 | train_loss: 0.374 | test_loss: 0.363\n",
      "Epoch 12/60 | Batch 0/77 | train_loss: 0.345 | test_loss: 0.347\n",
      "Epoch 12/60 | Batch 50/77 | train_loss: 0.320 | test_loss: 0.314\n",
      "Epoch 13/60 | Batch 0/77 | train_loss: 0.298 | test_loss: 0.306\n",
      "Epoch 13/60 | Batch 50/77 | train_loss: 0.276 | test_loss: 0.278\n",
      "Epoch 14/60 | Batch 0/77 | train_loss: 0.255 | test_loss: 0.268\n",
      "Epoch 14/60 | Batch 50/77 | train_loss: 0.247 | test_loss: 0.240\n",
      "Epoch 15/60 | Batch 0/77 | train_loss: 0.225 | test_loss: 0.231\n",
      "Epoch 15/60 | Batch 50/77 | train_loss: 0.214 | test_loss: 0.214\n",
      "Epoch 16/60 | Batch 0/77 | train_loss: 0.198 | test_loss: 0.204\n",
      "Epoch 16/60 | Batch 50/77 | train_loss: 0.184 | test_loss: 0.184\n",
      "Epoch 17/60 | Batch 0/77 | train_loss: 0.175 | test_loss: 0.185\n",
      "Epoch 17/60 | Batch 50/77 | train_loss: 0.163 | test_loss: 0.164\n",
      "Epoch 18/60 | Batch 0/77 | train_loss: 0.152 | test_loss: 0.170\n",
      "Epoch 18/60 | Batch 50/77 | train_loss: 0.140 | test_loss: 0.152\n",
      "Epoch 19/60 | Batch 0/77 | train_loss: 0.129 | test_loss: 0.147\n",
      "Epoch 19/60 | Batch 50/77 | train_loss: 0.130 | test_loss: 0.131\n",
      "Epoch 20/60 | Batch 0/77 | train_loss: 0.114 | test_loss: 0.130\n",
      "Epoch 20/60 | Batch 50/77 | train_loss: 0.111 | test_loss: 0.125\n",
      "Epoch 21/60 | Batch 0/77 | train_loss: 0.103 | test_loss: 0.120\n",
      "Epoch 21/60 | Batch 50/77 | train_loss: 0.096 | test_loss: 0.107\n",
      "Epoch 22/60 | Batch 0/77 | train_loss: 0.091 | test_loss: 0.106\n",
      "Epoch 22/60 | Batch 50/77 | train_loss: 0.093 | test_loss: 0.103\n",
      "Epoch 23/60 | Batch 0/77 | train_loss: 0.080 | test_loss: 0.090\n",
      "Epoch 23/60 | Batch 50/77 | train_loss: 0.072 | test_loss: 0.089\n",
      "Epoch 24/60 | Batch 0/77 | train_loss: 0.069 | test_loss: 0.083\n",
      "Epoch 24/60 | Batch 50/77 | train_loss: 0.065 | test_loss: 0.074\n",
      "Epoch 25/60 | Batch 0/77 | train_loss: 0.062 | test_loss: 0.076\n",
      "Epoch 25/60 | Batch 50/77 | train_loss: 0.058 | test_loss: 0.068\n",
      "Epoch 26/60 | Batch 0/77 | train_loss: 0.054 | test_loss: 0.072\n",
      "Epoch 26/60 | Batch 50/77 | train_loss: 0.052 | test_loss: 0.065\n",
      "Epoch 27/60 | Batch 0/77 | train_loss: 0.046 | test_loss: 0.062\n",
      "Epoch 27/60 | Batch 50/77 | train_loss: 0.052 | test_loss: 0.056\n",
      "Epoch 28/60 | Batch 0/77 | train_loss: 0.042 | test_loss: 0.060\n",
      "Epoch 28/60 | Batch 50/77 | train_loss: 0.046 | test_loss: 0.054\n",
      "Epoch 29/60 | Batch 0/77 | train_loss: 0.042 | test_loss: 0.048\n",
      "Epoch 29/60 | Batch 50/77 | train_loss: 0.044 | test_loss: 0.049\n",
      "Epoch 30/60 | Batch 0/77 | train_loss: 0.035 | test_loss: 0.044\n",
      "Epoch 30/60 | Batch 50/77 | train_loss: 0.037 | test_loss: 0.050\n",
      "Epoch 31/60 | Batch 0/77 | train_loss: 0.029 | test_loss: 0.038\n",
      "Epoch 31/60 | Batch 50/77 | train_loss: 0.033 | test_loss: 0.045\n",
      "Epoch 32/60 | Batch 0/77 | train_loss: 0.027 | test_loss: 0.037\n",
      "Epoch 32/60 | Batch 50/77 | train_loss: 0.029 | test_loss: 0.041\n",
      "Epoch 33/60 | Batch 0/77 | train_loss: 0.024 | test_loss: 0.037\n",
      "Epoch 33/60 | Batch 50/77 | train_loss: 0.028 | test_loss: 0.039\n",
      "Epoch 34/60 | Batch 0/77 | train_loss: 0.024 | test_loss: 0.034\n",
      "Epoch 34/60 | Batch 50/77 | train_loss: 0.027 | test_loss: 0.034\n",
      "Epoch 35/60 | Batch 0/77 | train_loss: 0.020 | test_loss: 0.032\n",
      "Epoch 35/60 | Batch 50/77 | train_loss: 0.022 | test_loss: 0.034\n",
      "Epoch 36/60 | Batch 0/77 | train_loss: 0.020 | test_loss: 0.030\n",
      "Epoch 36/60 | Batch 50/77 | train_loss: 0.021 | test_loss: 0.030\n",
      "Epoch 37/60 | Batch 0/77 | train_loss: 0.018 | test_loss: 0.031\n",
      "Epoch 37/60 | Batch 50/77 | train_loss: 0.019 | test_loss: 0.028\n",
      "Epoch 38/60 | Batch 0/77 | train_loss: 0.019 | test_loss: 0.027\n",
      "Epoch 38/60 | Batch 50/77 | train_loss: 0.019 | test_loss: 0.028\n",
      "Epoch 39/60 | Batch 0/77 | train_loss: 0.015 | test_loss: 0.024\n",
      "Epoch 39/60 | Batch 50/77 | train_loss: 0.017 | test_loss: 0.025\n",
      "Epoch 40/60 | Batch 0/77 | train_loss: 0.015 | test_loss: 0.024\n",
      "Epoch 40/60 | Batch 50/77 | train_loss: 0.014 | test_loss: 0.019\n",
      "Epoch 41/60 | Batch 0/77 | train_loss: 0.012 | test_loss: 0.018\n",
      "Epoch 41/60 | Batch 50/77 | train_loss: 0.014 | test_loss: 0.025\n",
      "Epoch 42/60 | Batch 0/77 | train_loss: 0.012 | test_loss: 0.021\n",
      "Epoch 42/60 | Batch 50/77 | train_loss: 0.014 | test_loss: 0.021\n",
      "Epoch 43/60 | Batch 0/77 | train_loss: 0.010 | test_loss: 0.018\n",
      "Epoch 43/60 | Batch 50/77 | train_loss: 0.016 | test_loss: 0.017\n",
      "Epoch 44/60 | Batch 0/77 | train_loss: 0.010 | test_loss: 0.020\n",
      "Epoch 44/60 | Batch 50/77 | train_loss: 0.012 | test_loss: 0.017\n",
      "Epoch 45/60 | Batch 0/77 | train_loss: 0.010 | test_loss: 0.016\n",
      "Epoch 45/60 | Batch 50/77 | train_loss: 0.010 | test_loss: 0.016\n",
      "Epoch 46/60 | Batch 0/77 | train_loss: 0.008 | test_loss: 0.014\n",
      "Epoch 46/60 | Batch 50/77 | train_loss: 0.010 | test_loss: 0.013\n",
      "Epoch 47/60 | Batch 0/77 | train_loss: 0.008 | test_loss: 0.014\n",
      "Epoch 47/60 | Batch 50/77 | train_loss: 0.011 | test_loss: 0.017\n",
      "Epoch 48/60 | Batch 0/77 | train_loss: 0.009 | test_loss: 0.017\n",
      "Epoch 48/60 | Batch 50/77 | train_loss: 0.013 | test_loss: 0.013\n",
      "Epoch 49/60 | Batch 0/77 | train_loss: 0.007 | test_loss: 0.013\n",
      "Epoch 49/60 | Batch 50/77 | train_loss: 0.008 | test_loss: 0.012\n",
      "Epoch 50/60 | Batch 0/77 | train_loss: 0.007 | test_loss: 0.013\n",
      "Epoch 50/60 | Batch 50/77 | train_loss: 0.007 | test_loss: 0.014\n",
      "Epoch 51/60 | Batch 0/77 | train_loss: 0.006 | test_loss: 0.011\n",
      "Epoch 51/60 | Batch 50/77 | train_loss: 0.007 | test_loss: 0.011\n",
      "Epoch 52/60 | Batch 0/77 | train_loss: 0.006 | test_loss: 0.011\n",
      "Epoch 52/60 | Batch 50/77 | train_loss: 0.006 | test_loss: 0.010\n",
      "Epoch 53/60 | Batch 0/77 | train_loss: 0.005 | test_loss: 0.011\n",
      "Epoch 53/60 | Batch 50/77 | train_loss: 0.006 | test_loss: 0.009\n",
      "Epoch 54/60 | Batch 0/77 | train_loss: 0.005 | test_loss: 0.014\n",
      "Epoch 54/60 | Batch 50/77 | train_loss: 0.006 | test_loss: 0.010\n",
      "Epoch 55/60 | Batch 0/77 | train_loss: 0.005 | test_loss: 0.010\n",
      "Epoch 55/60 | Batch 50/77 | train_loss: 0.005 | test_loss: 0.009\n",
      "Epoch 56/60 | Batch 0/77 | train_loss: 0.005 | test_loss: 0.011\n",
      "Epoch 56/60 | Batch 50/77 | train_loss: 0.005 | test_loss: 0.010\n",
      "Epoch 57/60 | Batch 0/77 | train_loss: 0.004 | test_loss: 0.009\n",
      "Epoch 57/60 | Batch 50/77 | train_loss: 0.005 | test_loss: 0.011\n",
      "Epoch 58/60 | Batch 0/77 | train_loss: 0.006 | test_loss: 0.017\n",
      "Epoch 58/60 | Batch 50/77 | train_loss: 0.027 | test_loss: 0.163\n",
      "Epoch 59/60 | Batch 0/77 | train_loss: 0.013 | test_loss: 0.022\n",
      "Epoch 59/60 | Batch 50/77 | train_loss: 0.007 | test_loss: 0.010\n",
      "Epoch 60/60 | Batch 0/77 | train_loss: 0.005 | test_loss: 0.008\n",
      "Epoch 60/60 | Batch 50/77 | train_loss: 0.005 | test_loss: 0.008\n",
      "IN: c o m m o n\n",
      "OUT: c m m n o o <EOS>\n",
      "\n",
      "IN: a p p l e\n",
      "OUT: a e l p p <EOS>\n",
      "\n",
      "IN: z h e d o n g\n",
      "OUT: d e g h n o z <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from seq2seq_beam import Seq2Seq\n",
    "import sys\n",
    "if int(sys.version[0]) == 2:\n",
    "    from io import open\n",
    "\n",
    "\n",
    "def read_data(path):\n",
    "    with open(path, 'r', encoding='utf-8') as f:\n",
    "        return f.read()\n",
    "# end function read_data\n",
    "\n",
    "\n",
    "def build_map(data):\n",
    "    specials = ['<GO>',  '<EOS>', '<PAD>', '<UNK>']\n",
    "    chars = list(set([char for line in data.split('\\n') for char in line]))\n",
    "    idx2char = {idx: char for idx, char in enumerate(specials + chars)}\n",
    "    char2idx = {char: idx for idx, char in idx2char.items()}\n",
    "    return idx2char, char2idx\n",
    "# end function build_map\n",
    "\n",
    "\n",
    "def preprocess_data():\n",
    "    X_data = read_data('temp/letters_source.txt')\n",
    "    Y_data = read_data('temp/letters_target.txt')\n",
    "\n",
    "    X_idx2char, X_char2idx = build_map(X_data)\n",
    "    Y_idx2char, Y_char2idx = build_map(Y_data)\n",
    "\n",
    "    x_unk = X_char2idx['<UNK>']\n",
    "    y_unk = Y_char2idx['<UNK>']\n",
    "    y_eos = Y_char2idx['<EOS>']\n",
    "\n",
    "    X_indices = [[X_char2idx.get(char, x_unk) for char in line] for line in X_data.split('\\n')]\n",
    "    Y_indices = [[Y_char2idx.get(char, y_unk) for char in line] + [y_eos] for line in Y_data.split('\\n')]\n",
    "\n",
    "    return X_indices, Y_indices, X_char2idx, Y_char2idx, X_idx2char, Y_idx2char\n",
    "# end function preprocess_data\n",
    "\n",
    "\n",
    "def main():\n",
    "    BATCH_SIZE = 128\n",
    "    X_indices, Y_indices, X_char2idx, Y_char2idx, X_idx2char, Y_idx2char = preprocess_data()\n",
    "    X_train = X_indices[BATCH_SIZE:]\n",
    "    Y_train = Y_indices[BATCH_SIZE:]\n",
    "    X_test = X_indices[:BATCH_SIZE]\n",
    "    Y_test = Y_indices[:BATCH_SIZE]\n",
    "\n",
    "    model = Seq2Seq(\n",
    "        rnn_size = 50,\n",
    "        n_layers = 2,\n",
    "        X_word2idx = X_char2idx,\n",
    "        encoder_embedding_dim = 15,\n",
    "        Y_word2idx = Y_char2idx,\n",
    "        decoder_embedding_dim = 15,\n",
    "    )\n",
    "    model.fit(X_train, Y_train, val_data=(X_test, Y_test), batch_size=BATCH_SIZE)\n",
    "    model.infer('common', X_idx2char, Y_idx2char)\n",
    "    model.infer('apple', X_idx2char, Y_idx2char)\n",
    "    model.infer('zhedong', X_idx2char, Y_idx2char)\n",
    "# end function main\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
